 /*
  *@(#)AdSpiderImpl.java		8 Sept. 2000
  *
  * Copyright: MetaPlus Inc & Inquiry Management Systems Ltd.  2000
  *
  * This software is the proprietary information of MetaPlus Inc & 
  * Inquiry Management Systems Ltd.  
  * Use is subject to license terms.
  */
package com.ims.iaa;

import java.io.*; 
import java.net.*; 
import java.util.*; 
import java.sql.*;
import java.rmi.*;
import java.rmi.server.*;

import com.ims.iaa.html.*;

/** 
  *
  * <code>AdSpiderImpl</code> objects represents ad spiders which are controlled 
  * by the Spider Control Center to perform search request. 
  * This class implements the
  * <code>AdSpider</code> remote interface to support spider behaviors, such as 
  * can be notified to perform ads search request, cancel the process of request, 
  * terminate the execution or get/set its name. The implementation of the 
  * <code>Rnunnable</code> interface makes the spider able to start a new thread 
  * when perform search request.
  *
  * @see AdSpider
  *
  * @author Anthony Zhang
  *
  * @version 1.00, 25 Sept. 2000
  */
public class AdSpiderImpl  implements AdSpider, Runnable {

	private static final int WAIT_IMPANALYZER = 1000;
	//The spider number in one Virtual Machine
	static int spiderCount = 0;
	
	//record the last active time in System.currentTimeMillis() format
	private long lastActiveTime = 0;
	
	//indicate the health of the spider
	private volatile boolean healthState = true;
	
	/**
	  * The spider name used to distingush different spiders
	  */
	private String spiderName;

	/**
	 * True for this flag means this is a new spiders and have no assign name 
	 * by server yet, while false means it is already assigned name by the server.
	 */ 
	private boolean isNewSpider = true;
	
	/** The working thread used to process web search request.
	  * If this thread is null, it means the spider has nothing to do. 
	  */
	private Thread spiderThread = null;
	
	/**
	  * The remote request object the spider was working on.
	  */  
	private AdTrackRequest adReq = null;
	
	/**
	  * The local version of the request object the spider was working on.
	  */  
	private LocalRequest lReq = null;
	
	/** 
	  * SpiderDataStream is used to handle data which spider collectted from the web.
	  */	
	private SpiderDataStream sStream = null;
	
	private SpiderCtl spiderCenter;
	
	private ImpAnalyzerCtrl m_impAnalyzerCtrl = null;
	
	String hostName = "localhost";
	int registryPort = 1099; 
	
	/**
	  * when spider construct, it get the spider name, export itself as a RMI object
	  * which is capable of callback and tell the server it is free.
	  */
	public AdSpiderImpl(String host, int port, int iBatchIndex) {		
		//get the spider name
		setSpiderName(iBatchIndex);

		if ( host.length() != 0) {
			hostName = host;
		}				
		if ( port != 0 ) {
			registryPort = port;
		}
		//lookup the Spider Control Center	
		spiderCenter = IaaUtil.getSpiderCenter(hostName,registryPort);
		log("Spider Control Center created.");
		setActiveTime();
		
		//export RMI object
		exportObject();
		m_impAnalyzerCtrl = new ImpAnalyzerCtrl();
		//tell the server it is free
		raiseFreeEvent();
		
	}
	
	private void setActiveTime()  {
		if(healthState)lastActiveTime = System.currentTimeMillis();
	}
	
	private void setActiveTime(long activeTime)  {
		if(healthState)lastActiveTime = activeTime;
	}
	
	public void setHealthState(boolean flag)  {
		healthState = flag;
	}
	
	
	private void exportObject()  {
		try {
			if(IaaDebug.Debug)log("exporting to server." );
		
	    	UnicastRemoteObject.exportObject( this );			

		} catch (Exception e)  {
			log("error export the spider " + e);
		}
	}
	
	/**
	  * This method is called by server to assign unique number to the spider
	  */
	public void setSpiderName()  {
		if (isNewSpider ) {
			String localName = spiderName;
			spiderName = spiderName + "-" + spiderCount;
			isNewSpider = false;
			IaaUtil.log("Spider ["+ localName +"] got new name as ["+ spiderName +"].");
		}
	}
	
	private synchronized void setSpiderName(int iBatchIndex)  {
		spiderCount++;
		String localName = null;
		try  {
			localName = java.net.InetAddress.getLocalHost().getHostName();
		} catch (Exception e)  {
			IaaUtil.log("Error get localhost name" + e);
		}
		spiderName = localName + "_" + iBatchIndex + "_" + spiderCount;	
	}
	
	public String getSpiderName() {
		return spiderName;
	}
	
	public boolean isNewSpider() 
		throws RemoteException {
		return isNewSpider;
	}

    /**
      * This method is called by the Spider Control Center to notify the spider
	  * to perform the specific request.
	  *
	  * @param adTrackReq The request need to be preformed by the spider.
	  *
      * @return Boolean flag indicate succefully notified or not
      *
	  * @see AdSpider#notifyPerform
      */	
	public boolean notifyPerform(AdTrackRequest adTrackReq) {
		IaaDebug.begin(5, spiderName + " notified to perform request.");
		setActiveTime();
		adReq = adTrackReq;
		try  {
			adReq.setStartTime(	new Timestamp(  System.currentTimeMillis() ) );
			adReq.setSpiderName(spiderName);
			lReq = adReq.getLocalRequest();
			log(" notified to perform request ["+ lReq +"].");

		} catch (Exception e) {
			log("error contact the remote Request object " + e);		
			return false;
		}
		IaaDebug.end(5, spiderName + " notified to perform request.");
		raisePerformEvent();
		return true;
	}
	
	public boolean isBusy()  {
		if (adReq!=null) {
			return true;
		} else  { 
			return false;
		}
	}
	
	public LocalRequest getLocalRequest()  {
		return lReq;
	}
	
	public void clearSearch()  {
		log("clear search state");
		if (sStream!=null) {
			log("SpiderDataStream[ "+sStream+"]"); 
			//we need not set it here since we set it below
			//sStream.setSearchEnd();			
			sStream = null;
		}
		try  {
			//we set state and endtime here
			adReq.setSearchState(-10);
			adReq.setEndTime(
				new Timestamp(  System.currentTimeMillis() ) );
		} catch (Exception e)  {
			log("error set clear search state for the request " + e);
		}
		adReq = null;
		if (spiderThread!=null) {
			log("WorkingThread["+spiderThread+"]"); 
			spiderThread = null;
		}
		if (lReq!=null) {
			log("Search["+lReq+"]"); 
			lReq = null;	
		}
		log("LastActivetime["+ new Timestamp(getLastActiveTime())+"].");
	}
	
	public long getLastActiveTime()  {
		return m_impAnalyzerCtrl.getLastActiveTime();//lastActiveTime;
	}

	public void notifyTerminate()  {
		if (adReq!= null)  {
			if (sStream!= null) {
				sStream.setSearchEnd();
			} else  {
				log("error of spider status: request not null while datastream null");
			}
			try  {
				adReq.setSearchState(-1);
			} catch (Exception e)  {
				log("error contact the remote Request object " + e);
			}
		}
		log("inform the server of termination.");		
		try {
			if ( spiderCenter == null)  {
				log("can not find the Spider Control Center" );
				return;
			}
	       	spiderCenter.removeAdSpider(this);
			log("removed from the server spiders queue." );
		 } catch ( RemoteException re ) {
		    log("" + re );
		 }
	}			
	
/*	public void notifyCancel(AdTrackRequest adTrackReq) {
		if(IaaDebug.Debug) log("notified cancel request by Spider Control Center");
		try  {
			if (adReq.equals(adTrackReq)) {
				//set the search state to -1 
				//which means the search was not finished
				adReq.setSearchState(-1);
				raiseCancelEvent();
			}
		} catch (Exception e)  {
			log("error contact the remote Request object " + e);
		}	
	}*/

	public void run()  {
		setActiveTime();
		performRequest();
		//wait for a while to prevent the bad health spider register to the server again
		try  {
			Thread.sleep(5000);
		} catch (Exception e)  {
		
		}
		if(healthState)raiseFreeEvent();	
	}

	/**
	  * This method is called by the notifyPerform method to perform the specific 
	  * request. Any spider can only handle one request a time, so it first check the 
	  *
	  * @param adTrackReq The request need to be preformed by the spider.
	  *
	  * @return Boolean flag indicate succefully notified or not
	  *
	  * @see AdSpider#notifyPerform
	  *
	  */
	protected void raisePerformEvent() {
		setActiveTime();
		IaaDebug.begin(spiderName + " start new thread.");
		if ( spiderThread == null)  {
			spiderThread = new IaaThread(this, spiderName );
			spiderThread.start();
			IaaDebug.end(spiderName + " start new thread.");
		} else  {
			//This should never happened since when receive the new request the 
			//variable spiderThread should be null.
			log("!!!!! error create new thread. Old "+ spiderThread + " alived.");
			SpecialExceptionLog.getInstance().println("System.exit(3)");
			System.exit(3);
		}	
	}

/*	protected void raiseCancelEvent() {
		spiderThread = null;
		//here is a problem if the process does not stoped quickly .
		raiseFreeEvent();
	}*/
	
	protected void raiseFreeEvent() {
	
		Thread freeEventThread = new IaaThread(spiderName + "_RaiseFreeEvent")  {
			public void run()  {
				log("available for processing requests.");		
				try {
					if ( spiderCenter == null)  {
						log("!!!!Fatal error: can not find the Spider Control Center. Exit at once");
						return;
					}
	       			if(IaaDebug.Debug)log("start the registration." );
	       			spiderCenter.registerAdSpider(AdSpiderImpl.this);
    	   			if(IaaDebug.Debug)log("finish the registration." );
		      	} catch ( RemoteException re ) {
		        	IaaUtil.log( "registration error " + re );
      			}
			} // end of run()
		} ; //end of anonymous class
		freeEventThread.start();
	}
		
	
	/**
	  * This method is called by the run method of the AdSpiderImpl.
	  *
	  * @return void
	  *
	  * @see AdSpider#notifyPerform
	  *
	  */
	private void performRequest() {
		/** SpiderDataStream is used to handle data which spider collectted from the web.
		  * The second parameter indicated wether store to database or not
		  */
		setActiveTime();
		sStream = new SpiderDataStream(adReq,IaaDebug.ResultStoreDB);
		//20 Nov 2000 Anthony Zhang the sStream should never be null here
		IaaDebug.assert_6(sStream!= null, spiderName + "!!!!sStream is null.");
		
		if (IaaDebug.ResultStoreDB)  {
			sStream.setSpiderDB(IaaUtil.getSpiderDBConnection(hostName,registryPort));
		}
		
		setActiveTime();
		sStream.setSearchBegin();

		//25 Oct 2000 Anthony Zhang		
		/** While processing search request, if the spider encounters any exceptions we 
		  * have not handled properly in other classes, we will handle them here.
		  */
		try  {
				setActiveTime();
				if (lReq.getRequestParams() == null)  {
					log("started normal request ["+lReq+"].");
					performNormalRequest();
				} else  {
					log("started search form request ["+lReq+"].");
					performSpecialRequest();
				}
				if (healthState)  {
					sStream.setSearchEnd();
				} else  {
					//TODO destroy all resource and delete it from spider center.
					log("processing timeout, current thread is set to null");
				}
		} catch (Throwable e) {
				//25 Oct 2000 Anthony Zhang
				/** In case of these exception, the spider will set the search time to 
			  	  * current time and the search state to 999, then the spider will 
			      * return gracefully to a free state and waiting for new job.
			  	  */
				//write stack trace info into log file  
				writeStackTrace(e);	
				try  {
					//set end time and state to the remote request object
					adReq.setEndTime(new Timestamp(  System.currentTimeMillis() ) );					
					adReq.setSearchState(999);				
				} catch (Exception ex ) {
					log("error set End Time " + e );
				}
		} finally 	{
			if (healthState)  {
				sStream = null;
				adReq = null;
				//21 Nov 2000 Anthony Zhang 
				/** SpiderThread must reset to null to make the spider
				  * capable of handle the next request.
				  */
				spiderThread = null;
				log("finished request ["+lReq+"].");
				lReq = null;
			}
		}
	}

	private void performNormalRequest() {
	
		setActiveTime();
		
		int pageProcessed =0;
		// this is a long run method so we need someway to stop it 
		Thread myThread = Thread.currentThread();

		//++ add by ShengMengBai at 2005-04-30
		//if obey robots.txt, create RobotsTXTAnalyzer 
		RobotsTXTAnalyzer robotsTxt = null;
		if(lReq.getObeyRobotsTXT()){
		    URL url = null;
            try {
                url = URLUtil.getURL(lReq.getUrl());
            } catch (MalformedURLException e) {
    			log("construct Spider error" + e );
    			return;
            }
            robotsTxt = new RobotsTXTAnalyzer(url, lReq.getUserAgentObject().getUserAgent());
		}
		//-- add by ShengMengBai at 2005-04-30 

		Enumeration as;
		try {
			as = new Spider( lReq.getUrl(),	System.out, lReq.getDepth() );
		} catch ( MalformedURLException e ) {
			log("construct Spider error" + e );
			return;
		}
		//++change by ShengMengBai at 2005-04-28
		((Spider)as).setUserAgentObject(lReq.getUserAgentObject());
		((Spider)as).setRobotsTXTAnalyzer(robotsTxt);
		//((Spider)as).setUserAgent(lReq.getUserAgent());
		//--change by ShengMengBai at 2005-04-28
		IaaDebug.say(spiderName+" performed "+pageProcessed+" pages, and "+ ((Spider)as).todo.size() + " pages left.");

		m_impAnalyzerCtrl.startAnalyze(robotsTxt, sStream);
	    
		//++ Daniel 11-10-06 For impressioncount
		int impressionsCollected = 0;
		
		while ( as.hasMoreElements() && (healthState) )
	      {
			setActiveTime();
			pageProcessed++;
			
			IaaDebug.boldSay(spiderName+" performed "+pageProcessed+" pages of the total "+((Spider)as).totalNumber+" pages.");
			//++change by ShengMengBai at 2005-04-28
			SpiderConnectionGetter getter = (SpiderConnectionGetter) as.nextElement();
			//--change by ShengMengBai at 2005-04-28

//++changsheng added for access log 2006-7-20
			/*sStream.LogAccess(AccessLog.SITE_STRUCTURE ,((Spider)as).al);*/
//--
			if ( getter == null) {
				IaaDebug.say("???One of the getter is null???");
				continue;
			}
			URLConnection uc = getter.getURLConnection();

		    String mimeType = uc.getContentType();
		    	
		    // check if mimeType contains "text/html"
			// revised by mliao: 10/12/2000:  
			// some cgi programs don't set the content type in http header. In such cases
			// the mimeType will be null, and I decide to let the PageAnalyzer to try it.
			//if ( mimeType.indexOf("text/html")!= -1)  
			if ( mimeType ==null || mimeType.indexOf("text/html")!= -1)  {

				sStream.setSpiderConnection(getter);
				
				//++Modify ShengMengBai at 2005-02-28
				//	cause: because we can't add too more ImpAnalyzer thread, so maybe get null.
				ImpAnalyzer impA = m_impAnalyzerCtrl.getImpAnalyzer();
				impA.setImpDepth(getter.getDepth() + 1);
				//change by ShengMeng Bai at 2005-04-28
				//impA.setUserAgent(lReq.getUserAgent());
				impA.setUserAgent(uc.getRequestProperty("User-Agent"));
				PageAnalyzer pageAnalyzer = new PageAnalyzer(impA, uc, lReq.getSpecialJS());
				IaaDebug.assert_6(pageAnalyzer!= null, spiderName + " pageAnalyzer is null");
				//pageAnalyzer.parse();
				Thread pageThread = new Thread(pageAnalyzer);
				pageThread.start();
				while(pageThread.isAlive()){
				    if( pageAnalyzer.checkStatus() == false  ){
				        pageThread.stop() ;
				        impA.analyzeOver() ;//++changsheng 2007-4-5,very important**   
				        IaaUtil.log( "one page has timeout!!");
				        break ;
				    }
				    try {
                        Thread.sleep(1000);
                    } catch (InterruptedException e1) {
                    }
				}
				//--Modify ShengMengBai at 2005-02-28

				//impressionsCollected += pageAnalyzer.getNumImpressions(); 
			    if(sStream.isReachMax()){
			        break ;
			    }
			}
			//URLConnection does not support disconnect
			//uc.disconnect();
			
			//if(impressionsCollected > lReq.getMaxImpressions()) {
			if(sStream.isReachMax()){
				IaaDebug.say(spiderName+" is stopping after " + impressionsCollected + " impressions collected.");
				break;
			}
			
			IaaDebug.say(spiderName+" performed "+pageProcessed+" pages, and "+ ((Spider)as).todo.size() + " pages left.");
	    }
		
		m_impAnalyzerCtrl.ImpressionOver();
		//waiting for all ad datas are analyze.
		while(!m_impAnalyzerCtrl.isMissionComplete()){
			setActiveTime(m_impAnalyzerCtrl.getLastActiveTime());
			try{
				Thread.sleep(WAIT_IMPANALYZER);
			}catch (InterruptedException e){
				//IaaUtil.printStackTrace(e);
			}
		}
		
		//since spiderThread never changed, it should always equals to myThread
	    //IaaDebug.assert(spiderThread == myThread, spiderName + "spiderThread changed???");		
	    // 21 Nov 2000 this statement is moved to the end of performRequest
		//spiderThread = null;
	}
		
	private void performSpecialRequest() {
		Thread myThread = Thread.currentThread();
	
	  	IaaDebug.say( spiderName + 	" start analyse special URL pages");
        setActiveTime();
        String	searchUrlStr = lReq.getUrl();
        String  searchStartParams = lReq.getStartParams();
        String  searchEndParams = lReq.getEndParams();
		
        //++ Daniel 11-10-06 For impressioncount
		int impressionsCollected = 0;
        
		RequestParameters params = lReq.getRequestParams();
		Hashtable hash = params.getParameter();
		setActiveTime();
		Enumeration  enumer = hash.keys();
          
		URLConnection uc = null;
        String searchString = null;
        
		//++ add by ShengMengBai at 2005-04-30
		//if obey robots.txt, create RobotsTXTAnalyzer 
		RobotsTXTAnalyzer robotsTxt = null;
		if(lReq.getObeyRobotsTXT()){
		    URL url = null;
            try {
                url = URLUtil.getURL(lReq.getUrl());
            } catch (MalformedURLException e) {
                //it is impossible
    			return;
            }
            robotsTxt = new RobotsTXTAnalyzer(url, lReq.getUserAgentObject().getUserAgent());
		}
		//-- add by ShengMengBai at 2005-04-30 

		m_impAnalyzerCtrl.startAnalyze(robotsTxt, sStream);
        
		while (enumer.hasMoreElements() && (healthState))  {
		
	        setActiveTime();
	        String keyword = (String)hash.get(enumer.nextElement());
			searchString = searchUrlStr + searchStartParams + keyword + 
					searchEndParams;
			try  {
				if (params.isGetMethod())  {
					if (IaaDebug.Debug)	 {
						IaaUtil.log( "The search string is" + searchString);
					}				
					uc =HttpConnectionHelper.sendGetRequest(searchString,
							 null, lReq.getUserAgentObject().getUserAgent());
				} else  {/*
					if (params.isPostMethod())  {
					URL searchURL = null;
						try  {			
							searchURL = URLUtil.getURL(lReq.getUrl());
						} catch ( MalformedURLException e ) {
							IaaUtil.log("Bad URL " + e );
							return;
						}
						uc =HttpConnectionHelper.sendPostRequest(searchURL,hash,
							lReq.getUserAgent());			
					}*/
				}//end of if (params.isGetMethod())
			} catch (IOException e)  {
				IaaUtil.log("Can not open URL " + e );		
			}//end of try
		
			if ( uc == null) {
				IaaDebug.say("???One of the uc is null???");
				continue;
			}
		
			String mimeType = uc.getContentType();
			// check if mimeType contains "text/html"
			// revised by mliao: 10/12/2000:  
			// some cgi programs don't set the content type in http header. In such cases
			// the mimeType will be null, and I decide to let the PageAnalyzer to try it.
			//if ( mimeType.indexOf("text/html")!= -1)  
			if ( mimeType ==null || mimeType.indexOf("text/html")!= -1)  
			{
				sStream.setURLConnection(uc);
				//++Modify ShengMengBai at 2005-02-28
				//	cause: because we can't add too more ImpAnalyzer thread, so maybe get null.
				ImpAnalyzer impA = m_impAnalyzerCtrl.getImpAnalyzer();
				impA.setImpDepth(1);
				//change by ShengMeng Bai at 2005-04-28
				//impA.setUserAgent(lReq.getUserAgent());
				impA.setUserAgent(uc.getRequestProperty("User-Agent"));
				PageAnalyzer pageAnalyzer = new PageAnalyzer(impA, uc, lReq.getSpecialJS());
				IaaDebug.assert_6(pageAnalyzer!= null, spiderName + "pageAnalyzer is null");
				pageAnalyzer.parse();
				//--Modify ShengMengBai at 2005-02-28
				
				impressionsCollected += pageAnalyzer.getNumImpressions();
			}
			//URLConnection does not support disconnect
			//uc.disconnect();
			
			if(impressionsCollected > lReq.getMaxImpressions()) {
				IaaDebug.say(spiderName+" is stopping after " + impressionsCollected + " impressions collected.");
				break;
			}
		}	
		// 21 Nov 2000 this statement is moved to the end of performRequest
		//spiderThread = null;
		m_impAnalyzerCtrl.ImpressionOver();
		//waiting for all ad datas are analyze.
		while(!m_impAnalyzerCtrl.isMissionComplete()){
			setActiveTime(m_impAnalyzerCtrl.getLastActiveTime());
			try{
				Thread.sleep(1000);
			}catch (InterruptedException e){
				//IaaUtil.printStackTrace(e);
			}
		}
	}	
	
	private void log(String s)  {
		IaaUtil.log(s);
	}
	
	//25 Oct 2000 Anthony Zhang
	//Write exception stack trace info into the log file
	private void writeStackTrace(Throwable th) {
		StringWriter sw = new StringWriter();
		PrintWriter pw = new PrintWriter(sw);
		th.printStackTrace(pw);
		String throwStr = sw.toString();
		log("----------------------------IMPORTANT ERROR LOG INFORMATION--------------------------");
		log("!!!!PLEASE REPORT THE FOLLOWING ERROR LOG INFORMATION TO THE TECHNICAL SUPPPORT TEAM!!!!");
		log("!!!! Error while process request [" + lReq + "] Exception Info: "+ throwStr);
		log("----------------------------END OF THE ERROR LOG INFORMATION--------------------------");
	}				

}
